{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75d0f0ff-2afc-4af8-b82a-993f1f3a82d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import json\n",
    "import time\n",
    "from collections import defaultdict\n",
    "from pytz import timezone\n",
    "from datetime import datetime\n",
    "from sys import getsizeof as getsize\n",
    "import networkx as nx\n",
    "\n",
    "from common_util import *\n",
    "from sn_util import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "834ba542-1a5a-4cad-8d78-22a523ea05df",
   "metadata": {},
   "outputs": [],
   "source": [
    "endpoint_edge_map = {\n",
    "    'http://localhost:8080/api/user/follow POST': [[0, 2], [1, 3], [1, 4], [1, 5], [2, 1], [2, 6], [2, 7], [6, 8], [7, 9]],\n",
    "    'http://localhost:8080/api/user/unfollow POST': [[0, 5], [4, 1], [4, 2], [4, 3], [5, 4], [5, 6], [5, 7], [6, 8], [7, 9]],\n",
    "    'http://localhost:8080/api/user/get_followee GET': [[0, 1], [1, 2], [1, 3], [1, 4]],\n",
    "    'http://localhost:8080/api/post/compose POST': [[0, 24], [1, 10], [2, 0], [2, 1], [2, 3], [2, 4], [2, 5], [2, 6], [2, 7], [3, 18], [4, 21], [5, 13], [6, 9], [7, 27], [8, 14], [9, 8], [11, 2], [13, 12], [14, 15], [14, 16], [14, 17], [18, 19], [18, 20], [19, 22], [20, 23], [27, 25], [27, 26]],\n",
    "    'http://localhost:8080/api/user/get_follower GET': [[0, 1], [1, 2], [1, 3], [1, 4]],\n",
    "    'http://localhost:8080/api/home-timeline/read?start=0&stop=100 GET': [[1, 0], [1, 3], [2, 1]],\n",
    "    'http://localhost:8080/api/user/register POST': [[0, 1]],\n",
    "    'http://localhost:8080/api/user/login POST': [[0, 1], [1, 2]],\n",
    "}\n",
    "endpoint_node_map = {\n",
    "    'http://localhost:8080/api/user/follow POST': {0: 'nginx-web-server:Follow_0',\n",
    "                                                   1: 'social-graph-service:follow_server_0',\n",
    "                                                   2: 'social-graph-service:follow_with_username_server_0',\n",
    "                                                   3: 'social-graph-service:mongo_update_client_0',\n",
    "                                                   4: 'social-graph-service:social_graph_mongo_update_client_0',\n",
    "                                                   5: 'social-graph-service:social_graph_redis_update_client_0',\n",
    "                                                   6: 'user-service:get_user_id_server_0',\n",
    "                                                   7: 'user-service:get_user_id_server_1',\n",
    "                                                   8: 'user-service:user_mmc_get_user_id_client_0',\n",
    "                                                   9: 'user-service:user_mmc_get_user_id_client_1'},\n",
    "    'http://localhost:8080/api/user/unfollow POST': {0: 'nginx-web-server:Unfollow_0',\n",
    "                                                     1: 'social-graph-service:social_graph_mongo_delete_client_0',\n",
    "                                                     2: 'social-graph-service:social_graph_mongo_delete_client_1',\n",
    "                                                     3: 'social-graph-service:social_graph_redis_update_client_0',\n",
    "                                                     4: 'social-graph-service:unfollow_server_0',\n",
    "                                                     5: 'social-graph-service:unfollow_with_username_server_0',\n",
    "                                                     6: 'user-service:get_user_id_server_0',\n",
    "                                                     7: 'user-service:get_user_id_server_1',\n",
    "                                                     8: 'user-service:user_mmc_get_user_id_client_0',\n",
    "                                                     9: 'user-service:user_mmc_get_user_id_client_1'},\n",
    "    'http://localhost:8080/api/user/get_followee GET': {0: 'nginx-web-server:GetFollowee_0',\n",
    "                                                        1: 'social-graph-service:get_followees_server_0',\n",
    "                                                        2: 'social-graph-service:social_graph_mongo_find_client_0',\n",
    "                                                        3: 'social-graph-service:social_graph_redis_get_client_0',\n",
    "                                                        4: 'social-graph-service:social_graph_redis_insert_client_0'},\n",
    "    'http://localhost:8080/api/post/compose POST': {0: 'compose-post-service:compose_creator_client_0',\n",
    "                                                    1: 'compose-post-service:compose_media_client_0',\n",
    "                                                    2: 'compose-post-service:compose_post_server_0',\n",
    "                                                    3: 'compose-post-service:compose_text_client_0',\n",
    "                                                    4: 'compose-post-service:compose_unique_id_client_0',\n",
    "                                                    5: 'compose-post-service:store_post_client_0',\n",
    "                                                    6: 'compose-post-service:write_home_timeline_client_0',\n",
    "                                                    7: 'compose-post-service:write_user_timeline_client_0',\n",
    "                                                    8: 'home-timeline-service:get_followers_client_0',\n",
    "                                                    9: 'home-timeline-service:write_home_timeline_server_0',\n",
    "                                                    10: 'media-service:compose_media_server_0',\n",
    "                                                    11: 'nginx-web-server:compose_post_client_0',\n",
    "                                                    12: 'post-storage-service:post_storage_mongo_insert_client_0',\n",
    "                                                    13: 'post-storage-service:store_post_server_0',\n",
    "                                                    14: 'social-graph-service:get_followers_server_0',\n",
    "                                                    15: 'social-graph-service:social_graph_mongo_find_client_0',\n",
    "                                                    16: 'social-graph-service:social_graph_redis_get_client_0',\n",
    "                                                    17: 'social-graph-service:social_graph_redis_insert_client_0',\n",
    "                                                    18: 'text-service:compose_text_server_0',\n",
    "                                                    19: 'text-service:compose_urls_client_0',\n",
    "                                                    20: 'text-service:compose_user_mentions_client_0',\n",
    "                                                    21: 'unique-id-service:compose_unique_id_server_0',\n",
    "                                                    22: 'url-shorten-service:compose_urls_server_0',\n",
    "                                                    23: 'user-mention-service:compose_user_mentions_server_0',\n",
    "                                                    24: 'user-service:compose_creator_server_0',\n",
    "                                                    25: 'user-timeline-service:write_user_timeline_mongo_insert_client_0',\n",
    "                                                    26: 'user-timeline-service:write_user_timeline_redis_update_client_0',\n",
    "                                                    27: 'user-timeline-service:write_user_timeline_server_0'},\n",
    "    'http://localhost:8080/api/user/get_follower GET': {0: 'nginx-web-server:GetFollower_0',\n",
    "                                                        1: 'social-graph-service:get_followers_server_0',\n",
    "                                                        2: 'social-graph-service:social_graph_mongo_find_client_0',\n",
    "                                                        3: 'social-graph-service:social_graph_redis_get_client_0',\n",
    "                                                        4: 'social-graph-service:social_graph_redis_insert_client_0'},\n",
    "    'http://localhost:8080/api/home-timeline/read?start=0&stop=100 GET': {\n",
    "        0: 'home-timeline-service:read_home_timeline_redis_find_client_0',\n",
    "        1: 'home-timeline-service:read_home_timeline_server_0', 2: 'nginx-web-server:Login_0',\n",
    "        3: 'post-storage-service:post_storage_read_posts_server_0'},\n",
    "    'http://localhost:8080/api/user/register POST': {0: 'nginx-web-server:RegisterUser_0',\n",
    "                                                     1: 'user-service:register_user_server_0'},\n",
    "    'http://localhost:8080/api/user/login POST': {0: 'nginx-web-server:Login_0', 1: 'user-service:login_server_0',\n",
    "                                                  2: 'user-service:user_mmc_get_client_0'},\n",
    "}\n",
    "service_list = ['home-timeline-service', 'post-storage-service', 'media-service', 'nginx-web-server', 'text-service', 'user-mention-service', 'social-graph-service', 'user-service', 'compose-post-service', 'url-shorten-service', 'unique-id-service', 'user-timeline-service']\n",
    "service_list.sort()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c1d635ce-82bd-4d60-8ae1-92e82a1f1c63",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkl_name = '../data/sn/sn_trace_1.pkl'\n",
    "npy_name = '../data/sn/sn_metric_all.npy'\n",
    "idx_name = '../data/sn/sn_time_index.pkl'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f218ad8d-0926-45cb-8246-baa16fbad46e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24113\n",
      "(12, 9970, 5)\n",
      "9970\n"
     ]
    }
   ],
   "source": [
    "# 加载数据\n",
    "with open(pkl_name, 'rb') as f:\n",
    "    trace_dataset = pickle.load(f)\n",
    "print(len(trace_dataset))\n",
    "\n",
    "all_metrics = np.load(npy_name)\n",
    "print(all_metrics.shape)\n",
    "\n",
    "with open(idx_name, 'rb') as f:\n",
    "    time_index = pickle.load(f)\n",
    "print(len(time_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bae6afab-3497-4203-8965-b9d43a9d1db4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 24, 5)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_trace_metric_window(trace_dataset[0], all_metrics, time_index, endpoint_node_map, service_list, 24).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7b346065-33de-4d0d-882f-ab275592b27c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 5)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trace_dataset[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "139e6bbd-0483-46ee-a233-496fb52a2dc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#origin: 24113\n",
      "#normal: 5162\n",
      "#abnormal: 19463\n",
      "#new abnormal: 1540\n",
      "ratio of new abnormal in all abnormal: 7.912449262703591%\n"
     ]
    }
   ],
   "source": [
    "limit = np.array([0.5, 1, 0.8, 999, 999])\n",
    "num_new_abnormal = 0\n",
    "num_abnormal = 0\n",
    "old_abnormal = 0\n",
    "multi = 10\n",
    "\n",
    "normal_dataset = []\n",
    "abnormal_new_dataset = []\n",
    "other_abnormal_dataset = []\n",
    "for trace_data in trace_dataset:\n",
    "    flag = True\n",
    "    \n",
    "    metric_window = get_trace_metric_window(trace_data, all_metrics, time_index,\n",
    "                        endpoint_node_map, service_list, 24)    # (node_num, window_size, feat_metric)\n",
    "    if metric_window is None:\n",
    "        continue\n",
    "        \n",
    "    t = np.mean(np.mean(metric_window, axis=0), axis=0)\n",
    "    tt = np.sum(t > limit)\n",
    "    if tt > 0:\n",
    "        if trace_data[1]['is_abnormal'] == 0:\n",
    "            trace_data[1]['is_abnormal'] = 1\n",
    "            for i in range(multi):\n",
    "                abnormal_new_dataset.append(trace_data)\n",
    "                num_new_abnormal += 1\n",
    "            flag = False\n",
    "        else:\n",
    "            old_abnormal += 1\n",
    "            flag = False\n",
    "    \n",
    "    if trace_data[1]['is_abnormal'] > 0:\n",
    "        num_abnormal += 1\n",
    "        if flag:\n",
    "            other_abnormal_dataset.append(trace_data)\n",
    "    else:\n",
    "        normal_dataset.append(trace_data)\n",
    "#print(num_abnormal, num_new_abnormal, old_abnormal, num_abnormal-num_new_abnormal)\n",
    "\n",
    "print(f'#origin: {len(trace_dataset)}')\n",
    "print(f'#normal: {len(normal_dataset)}')\n",
    "print(f'#abnormal: {len(other_abnormal_dataset) + len(abnormal_new_dataset)}')\n",
    "print(f'#new abnormal: {len(abnormal_new_dataset)}')\n",
    "print(f'ratio of new abnormal in all abnormal: {len(abnormal_new_dataset) / (len(other_abnormal_dataset) + len(abnormal_new_dataset)) * 100}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a3a37185-9b39-4dff-a9f5-e1879f39b4fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#origin: 10286\n",
      "#normal: 5162\n",
      "#abnormal: 5124\n",
      "#new abnormal: 1540\n",
      "ratio of new abnormal in all abnormal: 30.05464480874317%\n"
     ]
    }
   ],
   "source": [
    "dataset = normal_dataset + abnormal_new_dataset + other_abnormal_dataset[:int(0.2*len(other_abnormal_dataset))]\n",
    "print(f'#origin: {len(dataset)}')\n",
    "print(f'#normal: {len(normal_dataset)}')\n",
    "print(f'#abnormal: {int(0.2*len(other_abnormal_dataset)) + len(abnormal_new_dataset)}')\n",
    "print(f'#new abnormal: {len(abnormal_new_dataset)}')\n",
    "print(f'ratio of new abnormal in all abnormal: {len(abnormal_new_dataset) / (int(0.2*len(other_abnormal_dataset)) + len(abnormal_new_dataset)) * 100}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "afc45fa8-8392-400b-a30f-cca3e348db85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved to ../data/sn/sn_trace_new.pkl, dataset length: 10286\n"
     ]
    }
   ],
   "source": [
    "new_pkl_name = '../data/sn/sn_trace_new.pkl'\n",
    "if os.path.exists(new_pkl_name):\n",
    "    os.remove(new_pkl_name)\n",
    "\n",
    "with open(new_pkl_name, 'wb') as f:\n",
    "    pickle.dump(dataset, f)\n",
    "    \n",
    "print(\"saved to {}, dataset length: {}\".format(new_pkl_name, len(dataset)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0d2221e-55cf-4ec2-80ff-84ca099b2f95",
   "metadata": {},
   "source": [
    "ratio:\n",
    "\n",
    "1: 18784 150 679 18634\n",
    "4: 18767 133 676 18634\n",
    "16: 18777 143 700 18634\n",
    "24: 18783 154 706 18629\n",
    "48: 18776 183 717 18593"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "trace",
   "language": "python",
   "name": "trace"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
